{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "3d75b4c1",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from tensorflow.keras.models import Sequential\n",
    "from tensorflow.keras.layers import Dense, Flatten\n",
    "from tensorflow.keras.optimizers import Adam\n",
    "import pyarrow.parquet as pq\n",
    "from sklearn.model_selection import train_test_split\n",
    "import csv\n",
    "import subprocess\n",
    "import getpass\n",
    "import os\n",
    "import gzip\n",
    "from os import listdir\n",
    "from os.path import isfile, join\n",
    "from SciServer import Authentication\n",
    "import tensorflow as tf\n",
    "from tensorflow.keras.callbacks import ReduceLROnPlateau, EarlyStopping\n",
    "from tensorflow.keras.optimizers import Adam\n",
    "from tensorflow.keras.metrics import AUC"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "eecf0b25-5d8c-4d79-a361-d229fc0106ed",
   "metadata": {},
   "source": [
    "# Data Loading"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "9368f448",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_dataset = pd.read_csv(\"/home/idies/workspace/SAFE/MinooEmir/new_complete_features.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "4c31bdb5-8787-46c1-8738-11cd28dded51",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_dataset = df_dataset[['formatted_time', 'hf_original', 'hf_type_original', 'HDL',\n",
    "       'tot_cholesterol', 'glucose', 'bnp',\n",
    "       'Arterial Blood Pressure diastolic', 'Arterial Blood Pressure systolic',\n",
    "       'Heart Rate', 'gender', 'race', 'age']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "37191c06",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['formatted_time', 'hf_original', 'hf_type_original', 'HDL',\n",
       "       'tot_cholesterol', 'glucose', 'bnp',\n",
       "       'Arterial Blood Pressure diastolic', 'Arterial Blood Pressure systolic',\n",
       "       'Heart Rate', 'gender', 'race', 'age'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_dataset.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "31bab527-db57-49ee-af64-64877f72eb9c",
   "metadata": {},
   "outputs": [],
   "source": [
    "feature_list = ['HDL',\n",
    "               'tot_cholesterol', \n",
    "               'glucose', \n",
    "               'bnp',\n",
    "               'Arterial Blood Pressure diastolic', \n",
    "               'Arterial Blood Pressure systolic',\n",
    "               'Heart Rate', \n",
    "               'gender', \n",
    "               'race', \n",
    "               'age']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "a85aa509",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>hf_original</th>\n",
       "      <th>hf_type_original</th>\n",
       "      <th>HDL</th>\n",
       "      <th>tot_cholesterol</th>\n",
       "      <th>glucose</th>\n",
       "      <th>bnp</th>\n",
       "      <th>Arterial Blood Pressure diastolic</th>\n",
       "      <th>Arterial Blood Pressure systolic</th>\n",
       "      <th>Heart Rate</th>\n",
       "      <th>gender</th>\n",
       "      <th>race</th>\n",
       "      <th>age</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>formatted_time</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>13:45:00_11_01_2110_18106347</th>\n",
       "      <td>0</td>\n",
       "      <td>Non-HF</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>95.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>66.068966</td>\n",
       "      <td>108.655172</td>\n",
       "      <td>100.800000</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>48</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>04:02:00_17_01_2110_18780420</th>\n",
       "      <td>0</td>\n",
       "      <td>Non-HF</td>\n",
       "      <td>56.0</td>\n",
       "      <td>159.0</td>\n",
       "      <td>113.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>59.958333</td>\n",
       "      <td>1</td>\n",
       "      <td>7</td>\n",
       "      <td>84</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>02:02:00_22_01_2110_16006168</th>\n",
       "      <td>0</td>\n",
       "      <td>Non-HF</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>96.142857</td>\n",
       "      <td>NaN</td>\n",
       "      <td>80.428571</td>\n",
       "      <td>131.785714</td>\n",
       "      <td>92.785714</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17:21:00_30_01_2110_14816979</th>\n",
       "      <td>0</td>\n",
       "      <td>Non-HF</td>\n",
       "      <td>39.0</td>\n",
       "      <td>189.0</td>\n",
       "      <td>98.133333</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>96.173913</td>\n",
       "      <td>1</td>\n",
       "      <td>7</td>\n",
       "      <td>30</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17:07:00_01_02_2110_13956717</th>\n",
       "      <td>0</td>\n",
       "      <td>Non-HF</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>106.777778</td>\n",
       "      <td>NaN</td>\n",
       "      <td>52.473684</td>\n",
       "      <td>98.684211</td>\n",
       "      <td>80.157895</td>\n",
       "      <td>1</td>\n",
       "      <td>7</td>\n",
       "      <td>72</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                              hf_original hf_type_original   HDL  \\\n",
       "formatted_time                                                     \n",
       "13:45:00_11_01_2110_18106347            0           Non-HF   NaN   \n",
       "04:02:00_17_01_2110_18780420            0           Non-HF  56.0   \n",
       "02:02:00_22_01_2110_16006168            0           Non-HF   NaN   \n",
       "17:21:00_30_01_2110_14816979            0           Non-HF  39.0   \n",
       "17:07:00_01_02_2110_13956717            0           Non-HF   NaN   \n",
       "\n",
       "                              tot_cholesterol     glucose  bnp  \\\n",
       "formatted_time                                                   \n",
       "13:45:00_11_01_2110_18106347              NaN   95.000000  NaN   \n",
       "04:02:00_17_01_2110_18780420            159.0  113.000000  NaN   \n",
       "02:02:00_22_01_2110_16006168              NaN   96.142857  NaN   \n",
       "17:21:00_30_01_2110_14816979            189.0   98.133333  NaN   \n",
       "17:07:00_01_02_2110_13956717              NaN  106.777778  NaN   \n",
       "\n",
       "                              Arterial Blood Pressure diastolic  \\\n",
       "formatted_time                                                    \n",
       "13:45:00_11_01_2110_18106347                          66.068966   \n",
       "04:02:00_17_01_2110_18780420                                NaN   \n",
       "02:02:00_22_01_2110_16006168                          80.428571   \n",
       "17:21:00_30_01_2110_14816979                                NaN   \n",
       "17:07:00_01_02_2110_13956717                          52.473684   \n",
       "\n",
       "                              Arterial Blood Pressure systolic  Heart Rate  \\\n",
       "formatted_time                                                               \n",
       "13:45:00_11_01_2110_18106347                        108.655172  100.800000   \n",
       "04:02:00_17_01_2110_18780420                               NaN   59.958333   \n",
       "02:02:00_22_01_2110_16006168                        131.785714   92.785714   \n",
       "17:21:00_30_01_2110_14816979                               NaN   96.173913   \n",
       "17:07:00_01_02_2110_13956717                         98.684211   80.157895   \n",
       "\n",
       "                              gender  race  age  \n",
       "formatted_time                                   \n",
       "13:45:00_11_01_2110_18106347       0     6   48  \n",
       "04:02:00_17_01_2110_18780420       1     7   84  \n",
       "02:02:00_22_01_2110_16006168       1     2   20  \n",
       "17:21:00_30_01_2110_14816979       1     7   30  \n",
       "17:07:00_01_02_2110_13956717       1     7   72  "
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_dataset.set_index(\"formatted_time\", inplace=True)\n",
    "df_dataset.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "04f330ec-93af-41d8-8dcd-361b8e1df2e4",
   "metadata": {},
   "source": [
    "## Impute BNP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "29c0c1c7-e24c-45e7-961b-b3cb18289f8c",
   "metadata": {},
   "outputs": [],
   "source": [
    "import random\n",
    "# Function to generate random NT-proBNP values based on age\n",
    "def generate_nt_proBNP(age):\n",
    "    if age < 75:\n",
    "        return random.uniform(0, 125)  # For adults younger than 75 years\n",
    "    else:\n",
    "        return random.uniform(0, 450)  # For adults 75 years or older\n",
    "\n",
    "# Identify missing values in 'probnp'\n",
    "missing_values = df_dataset['bnp'].isnull()\n",
    "\n",
    "# Determine age of individuals with missing 'probnp' values (replace 'age_column' with the actual age column name)\n",
    "missing_age = df_dataset.loc[missing_values, 'age']\n",
    "\n",
    "# Generate random NT-proBNP values based on age\n",
    "imputed_values = missing_age.apply(generate_nt_proBNP)\n",
    "\n",
    "# Replace missing values in 'probnp' with generated values\n",
    "df_dataset.loc[missing_values, 'bnp'] = imputed_values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "d28f3c3f-b718-4f71-b3e4-086f5d9933ef",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>hf_original</th>\n",
       "      <th>hf_type_original</th>\n",
       "      <th>HDL</th>\n",
       "      <th>tot_cholesterol</th>\n",
       "      <th>glucose</th>\n",
       "      <th>bnp</th>\n",
       "      <th>Arterial Blood Pressure diastolic</th>\n",
       "      <th>Arterial Blood Pressure systolic</th>\n",
       "      <th>Heart Rate</th>\n",
       "      <th>gender</th>\n",
       "      <th>race</th>\n",
       "      <th>age</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>formatted_time</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>13:45:00_11_01_2110_18106347</th>\n",
       "      <td>0</td>\n",
       "      <td>Non-HF</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>95.000000</td>\n",
       "      <td>50.134772</td>\n",
       "      <td>66.068966</td>\n",
       "      <td>108.655172</td>\n",
       "      <td>100.800000</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>48</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>04:02:00_17_01_2110_18780420</th>\n",
       "      <td>0</td>\n",
       "      <td>Non-HF</td>\n",
       "      <td>56.0</td>\n",
       "      <td>159.0</td>\n",
       "      <td>113.000000</td>\n",
       "      <td>394.510415</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>59.958333</td>\n",
       "      <td>1</td>\n",
       "      <td>7</td>\n",
       "      <td>84</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>02:02:00_22_01_2110_16006168</th>\n",
       "      <td>0</td>\n",
       "      <td>Non-HF</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>96.142857</td>\n",
       "      <td>100.554047</td>\n",
       "      <td>80.428571</td>\n",
       "      <td>131.785714</td>\n",
       "      <td>92.785714</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17:21:00_30_01_2110_14816979</th>\n",
       "      <td>0</td>\n",
       "      <td>Non-HF</td>\n",
       "      <td>39.0</td>\n",
       "      <td>189.0</td>\n",
       "      <td>98.133333</td>\n",
       "      <td>24.524234</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>96.173913</td>\n",
       "      <td>1</td>\n",
       "      <td>7</td>\n",
       "      <td>30</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17:07:00_01_02_2110_13956717</th>\n",
       "      <td>0</td>\n",
       "      <td>Non-HF</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>106.777778</td>\n",
       "      <td>90.666634</td>\n",
       "      <td>52.473684</td>\n",
       "      <td>98.684211</td>\n",
       "      <td>80.157895</td>\n",
       "      <td>1</td>\n",
       "      <td>7</td>\n",
       "      <td>72</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                              hf_original hf_type_original   HDL  \\\n",
       "formatted_time                                                     \n",
       "13:45:00_11_01_2110_18106347            0           Non-HF   NaN   \n",
       "04:02:00_17_01_2110_18780420            0           Non-HF  56.0   \n",
       "02:02:00_22_01_2110_16006168            0           Non-HF   NaN   \n",
       "17:21:00_30_01_2110_14816979            0           Non-HF  39.0   \n",
       "17:07:00_01_02_2110_13956717            0           Non-HF   NaN   \n",
       "\n",
       "                              tot_cholesterol     glucose         bnp  \\\n",
       "formatted_time                                                          \n",
       "13:45:00_11_01_2110_18106347              NaN   95.000000   50.134772   \n",
       "04:02:00_17_01_2110_18780420            159.0  113.000000  394.510415   \n",
       "02:02:00_22_01_2110_16006168              NaN   96.142857  100.554047   \n",
       "17:21:00_30_01_2110_14816979            189.0   98.133333   24.524234   \n",
       "17:07:00_01_02_2110_13956717              NaN  106.777778   90.666634   \n",
       "\n",
       "                              Arterial Blood Pressure diastolic  \\\n",
       "formatted_time                                                    \n",
       "13:45:00_11_01_2110_18106347                          66.068966   \n",
       "04:02:00_17_01_2110_18780420                                NaN   \n",
       "02:02:00_22_01_2110_16006168                          80.428571   \n",
       "17:21:00_30_01_2110_14816979                                NaN   \n",
       "17:07:00_01_02_2110_13956717                          52.473684   \n",
       "\n",
       "                              Arterial Blood Pressure systolic  Heart Rate  \\\n",
       "formatted_time                                                               \n",
       "13:45:00_11_01_2110_18106347                        108.655172  100.800000   \n",
       "04:02:00_17_01_2110_18780420                               NaN   59.958333   \n",
       "02:02:00_22_01_2110_16006168                        131.785714   92.785714   \n",
       "17:21:00_30_01_2110_14816979                               NaN   96.173913   \n",
       "17:07:00_01_02_2110_13956717                         98.684211   80.157895   \n",
       "\n",
       "                              gender  race  age  \n",
       "formatted_time                                   \n",
       "13:45:00_11_01_2110_18106347       0     6   48  \n",
       "04:02:00_17_01_2110_18780420       1     7   84  \n",
       "02:02:00_22_01_2110_16006168       1     2   20  \n",
       "17:21:00_30_01_2110_14816979       1     7   30  \n",
       "17:07:00_01_02_2110_13956717       1     7   72  "
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_dataset.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4f9091d8-7165-459a-b712-2a5c51b21d61",
   "metadata": {},
   "source": [
    "## Data splitting into train, validation, and test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "fef29178-0b05-4d18-855a-a1cce79be6b9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "for HF vs Non-HF\n"
     ]
    }
   ],
   "source": [
    "print(\"for HF vs Non-HF\")\n",
    "X_id = df_dataset.index\n",
    "X = df_dataset[feature_list]\n",
    "y = df_dataset[\"hf_original\"]\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.3, random_state = 0, stratify = y)    \n",
    "\n",
    "X_train_id, X_test_id, _, _ = train_test_split(X_id, y, test_size = 0.3, random_state = 0, stratify = y)\n",
    "X_train_id, X_val_id, _, _ = train_test_split(X_train_id, y_train, test_size = 0.5, random_state = 0, stratify = y_train)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "804a0f83-d205-46cb-9e87-084e084372d8",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.impute import SimpleImputer\n",
    "\n",
    "imputer = SimpleImputer(strategy=\"median\")\n",
    "X_train_imputed = imputer.fit_transform(X_train)\n",
    "X_test_imputed = imputer.transform(X_test)\n",
    "\n",
    "scaler = StandardScaler()\n",
    "X_train_scaled = scaler.fit_transform(X_train_imputed)\n",
    "X_test_scaled = scaler.transform(X_test_imputed)\n",
    "\n",
    "X_train_scaled_df = pd.DataFrame(X_train_scaled, columns=X.columns, index=X_train.index)\n",
    "X_test_scaled_df = pd.DataFrame(X_test_scaled, columns=X.columns, index=X_test.index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "65b34553-e4b2-457f-b3ae-77773d7ac047",
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train_scaled_df, X_val, y_train, y_val = train_test_split(X_train_scaled_df, y_train, test_size = 0.5, random_state = 0, stratify = y_train) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "8c39702c",
   "metadata": {},
   "outputs": [],
   "source": [
    "base_dir = '/home/idies/workspace/SAFE/ecg_preprocessed'\n",
    "\n",
    "# Create the full file paths\n",
    "# waveform_files = [f for f in os.listdir(base_dir)]\n",
    "\n",
    "train_file_paths = [f\"{file_id}\" for file_id in X_train_id]\n",
    "val_file_paths = [f\"{file_id}\" for file_id in X_val_id]\n",
    "test_file_paths = [f\"{file_id}\" for file_id in X_test_id]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d8eaf48a-fb62-4e3a-9617-28a169108d37",
   "metadata": {},
   "source": [
    "## Aligning waveform and tabular data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "d909404a-597c-48fd-a0a8-49f98574ffc0",
   "metadata": {},
   "outputs": [],
   "source": [
    "def aligning_dataset(waveform_files, waveform_dir, tab_dataset_feature, tab_dataset_y):\n",
    "    waveform_paths = []\n",
    "    tabular_dat_list = []\n",
    "    label = []\n",
    "    \n",
    "    for i, waveform_file_name in enumerate(waveform_files):\n",
    "        patient_id = waveform_file_name\n",
    "        if patient_id in tab_dataset_feature.index:\n",
    "            waveform_paths.append(os.path.join(waveform_dir, patient_id))\n",
    "            tabular_dat_list.append(tab_dataset_feature.loc[patient_id].values)\n",
    "            # Assume labels are included in the tabular data\n",
    "            label.append(tab_dataset_y.loc[patient_id])\n",
    "\n",
    "    # Convert lists to numpy arrays\n",
    "    tabular_dat_arr = np.array(tabular_dat_list)\n",
    "    labels_arr = np.array(label)\n",
    "\n",
    "    return waveform_paths, tabular_dat_arr, labels_arr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "1045351a-5b64-4b26-8be8-d8e11e27f74a",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "train_waveform_paths, train_tab_data, train_labels = aligning_dataset(train_file_paths,base_dir, X_train_scaled_df, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "9fd52bb0-6ba3-43c5-8d9e-8c414683e248",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_waveform_paths, train_tab_data, train_labels = aligning_dataset(train_file_paths,base_dir, X_train_scaled_df, y_train)\n",
    "\n",
    "val_waveform_paths, val_tab_data, val_labels = aligning_dataset(val_file_paths,base_dir, X_val, y_val)\n",
    "\n",
    "test_waveform_paths, test_tab_data, test_labels = aligning_dataset(test_file_paths,base_dir, X_test_scaled_df, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d4ebde96-4a55-48e2-8c31-7d2832a93c70",
   "metadata": {},
   "source": [
    "## Create data generator and loader for NN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "941a3b55-f260-4eac-8a08-35e34126280f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define the data generator\n",
    "def ecg_data_generator(waveform_paths, tabular_data, labels):\n",
    "    for i in range(len(waveform_paths)):\n",
    "        try:\n",
    "            # Load the waveform data from Parquet\n",
    "            waveform_data = pq.read_table(waveform_paths[i]).to_pandas().values  # Ensure it's a numpy array\n",
    "            tabular_data_sample = tabular_data[i]  # Corresponding tabular data\n",
    "            label = labels[i]\n",
    "            yield (waveform_data, tabular_data_sample), label\n",
    "        except Exception as e:\n",
    "            print(f'Error loading {waveform_paths[i]}: {e}')\n",
    "            continue\n",
    "\n",
    "# Create TensorFlow datasets\n",
    "def create_dataset(waveform_paths, tabular_data, labels, batch_size):\n",
    "    dataset = tf.data.Dataset.from_generator(\n",
    "        lambda: ecg_data_generator(waveform_paths, tabular_data, labels),\n",
    "        output_signature=(\n",
    "            (tf.TensorSpec(shape=(5000, 12), dtype=tf.float32), tf.TensorSpec(shape=(tabular_data.shape[1],), dtype=tf.float32)),\n",
    "            tf.TensorSpec(shape=(), dtype=tf.int32)\n",
    "        )\n",
    "    )\n",
    "    return dataset.batch(batch_size).prefetch(tf.data.AUTOTUNE).repeat()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "fae1342d-d64d-4e98-a9b7-99dfd9c6e3c5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Shape of train_tab_data: (6414, 10)\n",
      "Shape of val_tab_data: (6414, 10)\n",
      "Shape of test_tab_data: (5499, 10)\n"
     ]
    }
   ],
   "source": [
    "\n",
    "print(\"Shape of train_tab_data:\", train_tab_data.shape)\n",
    "print(\"Shape of val_tab_data:\", val_tab_data.shape)\n",
    "print(\"Shape of test_tab_data:\", test_tab_data.shape)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "42b1eed4",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Shuffle and batch the dataset\n",
    "batch_size = 32\n",
    "\n",
    "train_dataset = create_dataset(train_waveform_paths,train_tab_data, train_labels, batch_size)\n",
    "\n",
    "val_dataset = create_dataset(val_waveform_paths,val_tab_data, val_labels, batch_size)\n",
    "\n",
    "test_dataset = create_dataset(test_waveform_paths,test_tab_data, test_labels, batch_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "8596da45-17d2-46fc-9851-862a56f69377",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Waveform data shape: (32, 5000, 12)\n",
      "Waveform data: tf.Tensor(\n",
      "[[[ 0.    -0.02  -0.02  ...  0.015  0.015 -0.02 ]\n",
      "  [ 0.    -0.015 -0.015 ...  0.     0.005 -0.02 ]\n",
      "  [ 0.    -0.015 -0.015 ... -0.005  0.    -0.02 ]\n",
      "  ...\n",
      "  [-0.035  0.     0.035 ... -0.025 -0.025 -0.04 ]\n",
      "  [-0.035 -0.01   0.025 ... -0.025 -0.025 -0.04 ]\n",
      "  [-0.04  -0.02   0.02  ... -0.025 -0.025 -0.04 ]]\n",
      "\n",
      " [[ 0.115  0.065 -0.04  ... -0.065 -0.04  -0.06 ]\n",
      "  [ 0.115  0.065 -0.04  ... -0.065 -0.04  -0.06 ]\n",
      "  [ 0.115  0.065 -0.04  ... -0.075 -0.04  -0.07 ]\n",
      "  ...\n",
      "  [-0.155 -0.025  0.14  ... -0.065  0.     0.05 ]\n",
      "  [-0.14  -0.025  0.125 ... -0.06   0.005  0.05 ]\n",
      "  [-0.125 -0.015  0.12  ... -0.045  0.005  0.06 ]]\n",
      "\n",
      " [[ 0.01   0.02   0.005 ... -0.01  -0.025 -0.025]\n",
      "  [ 0.     0.02   0.015 ... -0.01  -0.025 -0.025]\n",
      "  [-0.005  0.01   0.01  ... -0.01  -0.025 -0.025]\n",
      "  ...\n",
      "  [-0.01   0.     0.005 ... -0.005 -0.015  0.005]\n",
      "  [-0.01  -0.01  -0.005 ... -0.005 -0.015  0.   ]\n",
      "  [-0.01  -0.02  -0.005 ... -0.01  -0.005 -0.005]]\n",
      "\n",
      " ...\n",
      "\n",
      " [[ 0.115  0.12   0.02  ...  0.07   0.11   0.12 ]\n",
      "  [ 0.115  0.11   0.01  ...  0.085  0.105  0.115]\n",
      "  [ 0.115  0.1    0.    ...  0.08   0.1    0.11 ]\n",
      "  ...\n",
      "  [ 0.235  0.09  -0.13  ...  0.01   0.165  0.18 ]\n",
      "  [ 0.245  0.09  -0.14  ...  0.01   0.17   0.18 ]\n",
      "  [ 0.255  0.08  -0.16  ...  0.01   0.17   0.18 ]]\n",
      "\n",
      " [[-0.14  -0.06   0.08  ...  0.16   0.1    0.08 ]\n",
      "  [-0.14  -0.06   0.08  ...  0.16   0.1    0.08 ]\n",
      "  [-0.14  -0.06   0.08  ...  0.14   0.1    0.08 ]\n",
      "  ...\n",
      "  [ 0.     0.     0.02  ...  0.04   0.1    0.06 ]\n",
      "  [-0.02   0.     0.02  ...  0.04   0.1    0.06 ]\n",
      "  [-0.02   0.     0.02  ...  0.04   0.1    0.06 ]]\n",
      "\n",
      " [[-0.015 -0.025  0.005 ...  0.08   0.07  -0.105]\n",
      "  [-0.005 -0.02   0.    ...  0.07   0.065 -0.105]\n",
      "  [ 0.    -0.02  -0.005 ...  0.07   0.065 -0.095]\n",
      "  ...\n",
      "  [ 0.02   0.01   0.005 ... -0.015 -0.02   0.055]\n",
      "  [ 0.025  0.015  0.005 ... -0.015 -0.01   0.045]\n",
      "  [ 0.025  0.015  0.005 ...  0.     0.01   0.035]]], shape=(32, 5000, 12), dtype=float32)\n",
      "Tabular data shape: (32, 10)\n",
      "Tabular data: tf.Tensor(\n",
      "[[-0.0491658  -0.02708648 -0.8102236  -0.19346799  0.14608522 -1.052338\n",
      "  -0.02129529  0.8183277   0.10493675 -1.8555796 ]\n",
      " [-0.0491658  -0.02708648  2.3903716  -0.20117193 -0.19497527  0.6106511\n",
      "  -1.5609134   0.8183277   0.6712508   0.16660394]\n",
      " [-0.0491658  -0.02708648 -1.0914582  -0.14435449 -0.03066595 -0.09993993\n",
      "  -1.3255173   0.8183277   0.10493675  1.1145024 ]\n",
      " [-1.2210019  -3.000452    0.74731064 -0.19896938 -0.03066595 -0.09993993\n",
      "   1.0051168   0.8183277   0.10493675  1.0513092 ]\n",
      " [-0.0491658  -0.02708648 -0.21374875 -0.13294199 -0.03066595 -0.09993993\n",
      "  -0.68931144 -1.2220043   0.10493675  1.3672755 ]\n",
      " [-0.0491658  -0.02708648  2.600436   -0.19854188 -0.03066595 -0.09993993\n",
      "  -0.694083    0.8183277   0.6712508  -0.27574873]\n",
      " [-0.0491658  -0.02708648  0.5542471   2.211076   -0.05623373 -1.2624185\n",
      "  -0.20791568 -1.2220043   0.10493675 -0.5917149 ]\n",
      " [-0.0491658  -0.02708648 -0.35703972 -0.20453613  0.18796316 -0.8680196\n",
      "   1.3734558   0.8183277  -2.7266338   0.29299042]\n",
      " [-0.0491658  -0.02708648  0.6608972  -0.20076278 -0.32955885 -1.980879\n",
      "   0.78292674  0.8183277   0.6712508   0.7985363 ]\n",
      " [-0.0491658  -0.02708648  2.1344688  -0.19963117 -0.03066595 -0.09993993\n",
      "   1.7728052   0.8183277   0.10493675  0.04021746]\n",
      " [-0.0491658  -0.02708648  0.0481855  -0.21951619  0.33219984  0.529132\n",
      "  -0.6335211   0.8183277   0.10493675 -0.02297578]\n",
      " [-0.0491658  -0.02708648  0.27795237 -0.12948218 -0.03066595 -0.09993993\n",
      "  -1.6171932   0.8183277   0.6712508   0.67214984]\n",
      " [-0.0491658  -0.02708648 -0.7275075  -0.21083002 -0.08402479 -0.45265523\n",
      "  -0.17271228  0.8183277   0.10493675  0.22979717]\n",
      " [-0.0491658  -0.02708648 -0.26429746 -0.14674929 -0.4083465  -0.15407544\n",
      "  -0.80841756  0.8183277   0.6712508   1.1145024 ]\n",
      " [-0.0491658  -0.02708648 -0.5400177  -0.20104913  0.26822695  0.35372043\n",
      "   0.45506385 -1.2220043   0.6712508   0.1034107 ]\n",
      " [ 1.1226703  -0.7704278  -0.12337378  6.06838    -0.03066595 -0.09993993\n",
      "   0.3974077   0.8183277   0.6712508  -0.02297578]\n",
      " [-0.0491658  -0.02708648 -0.59209824 -0.2022269  -0.58886385  0.03560028\n",
      "  -0.6914321  -1.2220043  -0.46137735  0.41937688]\n",
      " [-0.0491658  -0.02708648  0.4525752  -0.139156   -0.03066595 -0.09993993\n",
      "  -0.10654687 -1.2220043   0.6712508   0.7985363 ]\n",
      " [-0.0491658  -0.02708648  0.8592626  -0.20349415 -0.20844325  0.83062315\n",
      "   0.46760446  0.8183277   0.10493675  0.22979717]\n",
      " [-0.0491658  -0.02708648 -0.5317461  -0.21627972 -0.39884198 -1.1774892\n",
      "  -0.46027735 -1.2220043   0.10493675  0.29299042]\n",
      " [-0.0491658  -0.02708648 -0.7789753  -0.20767443 -0.03066595 -0.09993993\n",
      "   0.29203612 -1.2220043   0.6712508   1.0513092 ]\n",
      " [-0.0491658  -0.02708648 -0.47108766 -0.14923199 -0.1194584  -0.62086916\n",
      "  -0.40779036 -1.2220043   0.6712508   0.86172956]\n",
      " [-0.0491658  -0.02708648 -0.6809414  -0.20556505 -0.03066595 -0.09993993\n",
      "  -0.53238064  0.8183277   0.6712508  -2.4243188 ]\n",
      " [-0.0491658  -0.02708648  0.8149504  -0.21908286 -0.03066595 -0.09993993\n",
      "   0.54238313  0.8183277   0.10493675 -2.4243188 ]\n",
      " [-0.0491658  -0.02708648 -0.16779537 -0.16945435 -0.81029797 -0.33537272\n",
      "  -0.9023226  -1.2220043   0.10493675  0.86172956]\n",
      " [-0.0491658  -0.02708648  0.37198007 -0.21423577  0.26947755 -2.149093\n",
      "  -1.7322786   0.8183277   0.10493675 -0.4021352 ]\n",
      " [-0.0491658  -0.02708648 -0.5124457  -0.21546255  0.22088288 -0.7980946\n",
      "   0.12048773 -1.2220043   0.10493675  0.73534304]\n",
      " [-0.0491658  -0.02708648  0.85696495 -0.19913253 -0.03066595 -0.09993993\n",
      "   1.8109776  -1.2220043   0.6712508  -1.160454  ]\n",
      " [-0.0491658  -0.02708648 -0.74574745  0.631239   -0.03066595 -0.09993993\n",
      "  -0.45009804  0.8183277   0.10493675  0.92492276]\n",
      " [-0.0491658  -0.02708648 -0.8973936  -0.19315417 -0.03066595 -0.09993993\n",
      "  -0.21234983  0.8183277   0.6712508   1.3040822 ]\n",
      " [-0.0491658  -0.02708648 -0.58240455  3.8171818  -0.03066595 -0.09993993\n",
      "  -0.07525497 -1.2220043   0.10493675 -0.52852166]\n",
      " [-0.0491658  -0.02708648  0.06656685 -0.20810334 -0.03066595 -0.09993993\n",
      "  -0.6677783   0.8183277   0.6712508   0.35618365]], shape=(32, 10), dtype=float32)\n",
      "Label data shape: (32,)\n",
      "Label data: tf.Tensor([0 1 0 1 0 0 0 1 0 0 1 0 0 0 1 1 1 0 0 0 0 0 0 0 1 0 0 0 1 0 1 1], shape=(32,), dtype=int32)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-06-20 21:15:30.124535: W tensorflow/core/framework/local_rendezvous.cc:404] Local rendezvous is aborting with status: OUT_OF_RANGE: End of sequence\n"
     ]
    }
   ],
   "source": [
    "# Inspect the data inside the train_dataset\n",
    "for (waveform, tabular), label in val_dataset.take(1):  # Only take the first batch for inspection\n",
    "    print(\"Waveform data shape:\", waveform.shape)\n",
    "    print(\"Waveform data:\", waveform)\n",
    "    print(\"Tabular data shape:\", tabular.shape)\n",
    "    print(\"Tabular data:\", tabular)\n",
    "    print(\"Label data shape:\", label.shape)\n",
    "    print(\"Label data:\", label)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c0049dc8-7c67-4c54-becc-1c4f4f777a4e",
   "metadata": {},
   "source": [
    "# Define model architecture"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "id": "336ece76",
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "from keras import backend as K\n",
    "from keras.layers import (Input, Dense, Conv1D, Dropout, MaxPooling1D, \n",
    "                          Activation, Lambda, BatchNormalization, Add,\n",
    "                          Flatten, Attention, MultiHeadAttention)\n",
    "from keras.optimizers import Adam\n",
    "from keras.models import Model\n",
    "from keras.metrics import AUC\n",
    "from keras.models import Model, Sequential\n",
    "\n",
    "from tensorflow.keras.layers import Input, Conv1D, BatchNormalization, Conv2D, MaxPooling2D, \\\n",
    "    ReLU, Reshape, GlobalAveragePooling1D, Dense, Concatenate, Dropout, concatenate, LeakyReLU, SpatialDropout1D, Attention\n",
    "import logging\n",
    "from tensorflow.keras.layers import Layer, Dense, MultiHeadAttention, LayerNormalization\n",
    "\n",
    "# PAPER: Screening for cardiac contractile dysfunction using an artificial intelligence–enabled electrocardiogram\n",
    "#        https://www.nature.com/articles/s41591-018-0240-2\n",
    "# SOURCE REPO: https://github.com/chrisby/DeepCardiology\n",
    "class Attia_et_al_CNN():\n",
    "    def __init__(self, \n",
    "                 filter_numbers=[16, 16, 32, 32, 64, 64], \n",
    "                 kernel_widths=[7, 7, 5, 5, 3, 3], \n",
    "                 pool_sizes=[2, 2, 4, 2, 2, 4], \n",
    "                 spatial_num_filters=64, \n",
    "                 dense_dropout_rate=0.2, \n",
    "                 spatial_dropout_rate=0.2,\n",
    "                 dense_units=[64, 32], \n",
    "                 use_spatial_layer=False,\n",
    "                 verbose=1,\n",
    "                 use_residual=True):\n",
    "\n",
    "        self.filter_numbers = filter_numbers\n",
    "        self.kernel_widths = kernel_widths\n",
    "        self.pool_sizes = pool_sizes\n",
    "        self.spatial_num_filters = spatial_num_filters\n",
    "        self.dense_dropout_rate = dense_dropout_rate\n",
    "        self.spatial_dropout_rate = spatial_dropout_rate\n",
    "        self.dense_units = dense_units\n",
    "        self.use_spatial_layer = use_spatial_layer\n",
    "        self.verbose = verbose\n",
    "        self.use_residual = use_residual\n",
    "\n",
    "        self.att = Attention()\n",
    "\n",
    "        self.model = None\n",
    "\n",
    "        if self.verbose == 0:\n",
    "            return\n",
    "        \n",
    "        print(\"Attia et al. CNN model initialized with the following parameters:\")\n",
    "        print(f\"  filter_numbers: {self.filter_numbers}\")\n",
    "        print(f\"  kernel_widths: {self.kernel_widths}\")\n",
    "        print(f\"  pool_sizes: {self.pool_sizes}\")\n",
    "        print(f\"  spatial_num_filters: {self.spatial_num_filters}\")\n",
    "        print(f\"  dense_dropout_rate: {self.dense_dropout_rate}\")\n",
    "        print(f\"  spatial_dropout_rate: {self.spatial_dropout_rate}\")\n",
    "        print(f\"  dense_units: {self.dense_units}\")\n",
    "        print(f\"  use_spatial_layer: {self.use_spatial_layer}\")\n",
    "        print(f\"  use_residual: {self.use_residual}\")\n",
    "    \n",
    "    def get_temporal_layer(self, N, k, p, input_layer):\n",
    "        c = Conv1D(N, k, padding='same', kernel_initializer='he_normal')(input_layer)\n",
    "        b = tf.keras.layers.BatchNormalization()(c)\n",
    "        a = Activation('relu')(b)\n",
    "        p = MaxPooling1D(pool_size=p)(a)\n",
    "        do = SpatialDropout1D(self.spatial_dropout_rate)(p)\n",
    "        return do\n",
    "\n",
    "    def get_temporal_layer_with_residual(self, N, k, p, input_layer):\n",
    "        # Main pathway\n",
    "        x = Conv1D(N, k, padding='same', kernel_initializer='he_normal')(input_layer)\n",
    "        x = SpatialDropout1D(self.spatial_dropout_rate)(x)\n",
    "        x = BatchNormalization()(x)\n",
    "        x = Activation('relu')(x)\n",
    "        \n",
    "        # Shortcut pathway\n",
    "        # Ensure the shortcut matches the dimension of the main pathway's output, adjust filters and stride as necessary\n",
    "        shortcut = Conv1D(N, 1, padding='same', kernel_initializer='he_normal')(input_layer)  # 1x1 conv for matching dimension\n",
    "        shortcut = BatchNormalization()(shortcut)  # Optional, for matching feature-wise statistics\n",
    "        \n",
    "        # Merging the shortcut with the main pathway\n",
    "        merged_output = Add()([x, shortcut])  # Element-wise addition\n",
    "\n",
    "        x = MaxPooling1D(pool_size=p)(merged_output)\n",
    "        \n",
    "        return x\n",
    "    \n",
    "    def get_spatial_layer(self, kernel_size, input_layer):\n",
    "        c = Conv1D(self.spatial_num_filters, kernel_size, padding='same', data_format=\"channels_first\", kernel_initializer='he_normal')(input_layer)\n",
    "        b = tf.keras.layers.BatchNormalization()(c)\n",
    "        a = Activation('relu')(b)\n",
    "        do = SpatialDropout1D(self.spatial_dropout_rate)(a)\n",
    "        return do\n",
    "    \n",
    "    def get_fully_connected_layer(self, units, input_layer):\n",
    "        d = Dense(units, kernel_initializer='he_normal')(input_layer)\n",
    "        b = tf.keras.layers.BatchNormalization()(d)\n",
    "        a = Activation('relu')(b)\n",
    "        do = Dropout(self.dense_dropout_rate)(a)\n",
    "        return do\n",
    "\n",
    "    def build(self, input_shape=(5000, 12)):\n",
    "        input_layer = Input(shape=input_shape)\n",
    "        last_layer = input_layer\n",
    "        \n",
    "        for i in range(len(self.pool_sizes)):\n",
    "            if self.use_residual:\n",
    "                temp_layer = self.get_temporal_layer_with_residual(self.filter_numbers[i], self.kernel_widths[i],\n",
    "                                            self.pool_sizes[i], last_layer)\n",
    "            else:\n",
    "                temp_layer = self.get_temporal_layer(self.filter_numbers[i], self.kernel_widths[i],\n",
    "                                            self.pool_sizes[i], last_layer)\n",
    "            last_layer = temp_layer\n",
    "        \n",
    "        if self.use_spatial_layer:\n",
    "            last_layer = self.get_spatial_layer(input_shape[1], last_layer)\n",
    "\n",
    "        last_layer = Flatten()(last_layer)\n",
    "\n",
    "        for i in range(len(self.dense_units)):\n",
    "            dense_layer = self.get_fully_connected_layer(self.dense_units[i], last_layer)\n",
    "            last_layer = dense_layer\n",
    "\n",
    "        output_layer = Dense(1, activation='sigmoid')(last_layer)\n",
    "        self.model = Model(inputs=input_layer, outputs=output_layer)\n",
    "\n",
    "        if self.verbose > 0:\n",
    "            print(self.model.summary())\n",
    "        return self.model\n",
    "\n",
    "class MultiHeadCrossAttention(Layer):\n",
    "    def __init__(self, embed_dim, num_heads):\n",
    "        super().__init__()\n",
    "        self.cross_attention = MultiHeadAttention(num_heads=num_heads, key_dim=embed_dim)\n",
    "        self.dense_proj = Dense(embed_dim, activation='relu')\n",
    "        self.layer_norm = LayerNormalization(epsilon=1e-6)\n",
    "\n",
    "        # Project input dimensions to match expected [batch_size, sequence_length, embed_dim]\n",
    "        self.query_projection = Dense(embed_dim)\n",
    "        self.value_projection = Dense(embed_dim)\n",
    "\n",
    "    def call(self, query, value):\n",
    "        # Ensure query and value match required shape [batch_size, seq_length, embed_dim]\n",
    "        query = self.query_projection(tf.expand_dims(query, axis=1))\n",
    "        value = self.value_projection(tf.expand_dims(value, axis=1))\n",
    "\n",
    "        attn_output = self.cross_attention(query=query, value=value, key=value)\n",
    "        attn_output = self.dense_proj(attn_output[:, 0, :])  # Reshape output if needed\n",
    "        output = self.layer_norm(query[:, 0, :] + attn_output)\n",
    "\n",
    "        return output\n",
    "    \n",
    "# PAPER: Screening for cardiac contractile dysfunction using an artificial intelligence–enabled electrocardiogram\n",
    "#        https://www.nature.com/articles/s41591-018-0240-2\n",
    "# SOURCE REPO: https://github.com/chrisby/DeepCardiology\n",
    "class Attia_et_al_fusion():\n",
    "    def __init__(self, \n",
    "                 filter_numbers=[16, 16, 32, 32, 64, 64], \n",
    "                 kernel_widths=[5, 5, 5, 3, 3, 3], \n",
    "                 pool_sizes=[2, 2, 4, 2, 2, 4], \n",
    "                 spatial_num_filters=64, \n",
    "                 dropout_rate=0.2, \n",
    "                 dense_units=[64, 32], \n",
    "                 fusion_strategy=\"concat\",\n",
    "                 use_waveforms=True,\n",
    "                 use_residual = True,\n",
    "                 spatial_dropout_rate=0.2,\n",
    "                 verbose=1):\n",
    "        \n",
    "        # Fusion strategy options\n",
    "        # concat, self_attn, cross_attn\n",
    "        # mlp, tab_mlp\n",
    "        \n",
    "        self.filter_numbers = filter_numbers\n",
    "        self.kernel_widths = kernel_widths\n",
    "        self.pool_sizes = pool_sizes\n",
    "        self.spatial_num_filters = spatial_num_filters\n",
    "        self.dropout_rate = dropout_rate\n",
    "        self.dense_units = dense_units\n",
    "        self.fusion_strategy = fusion_strategy\n",
    "        self.use_waveforms = use_waveforms\n",
    "        self.use_residual = use_residual\n",
    "        self.spatial_dropout_rate = spatial_dropout_rate\n",
    "\n",
    "        self.verbose = verbose\n",
    "\n",
    "        self.model = None\n",
    "\n",
    "        self.att = Attention()\n",
    "\n",
    "        if self.verbose == 0:\n",
    "            return\n",
    "        \n",
    "        print(\"Attia et al. CNN model initialized with the following parameters:\")\n",
    "        print(f\"  filter_numbers: {self.filter_numbers}\")\n",
    "        print(f\"  kernel_widths: {self.kernel_widths}\")\n",
    "        print(f\"  pool_sizes: {self.pool_sizes}\")\n",
    "        print(f\"  spatial_num_filters: {self.spatial_num_filters}\")\n",
    "        print(f\"  dropout_rate: {self.dropout_rate}\")\n",
    "        print(f\"  dense_units: {self.dense_units}\")\n",
    "        print(f\"  fusion_strategy: {self.fusion_strategy}\")\n",
    "        print(f\"  use_waveforms: {self.use_waveforms}\")\n",
    "        print(f\"  use_residual: {self.use_residual}\")\n",
    "\n",
    "\n",
    "    def create_mlp(self, input_shape):\n",
    "        mlp = tf.keras.Sequential([Dense(128, input_shape=(40,), activation='relu'),\n",
    "                    Dropout(self.dropout_rate),\n",
    "                    Dense(32, activation='relu'),\n",
    "                    Dropout(self.dropout_rate),\n",
    "                    Dense(8, activation='relu'),\n",
    "                    Dropout(self.dropout_rate),\n",
    "                    Dense(1, activation='sigmoid')])\n",
    "        return mlp\n",
    "\n",
    "    def create_attn(self, input_shape):\n",
    "        attn = Attention()\n",
    "        return attn\n",
    "\n",
    "    def get_temporal_layer(self, N, k, p, input_layer):\n",
    "        c = Conv1D(N, k, padding='same')(input_layer)\n",
    "        # c = SeparableConv1D(N, k, padding='same', activation='relu')(input_layer)\n",
    "        b = tf.keras.layers.BatchNormalization()(c)\n",
    "        a = Activation('relu')(b)\n",
    "        p = MaxPooling1D(pool_size=p)(a)\n",
    "        do = SpatialDropout1D(0.1)(p)\n",
    "        return do\n",
    "    \n",
    "    def get_temporal_layer_with_residual(self, N, k, p, input_layer):\n",
    "        # Main pathway\n",
    "        x = Conv1D(N, k, padding='same', kernel_initializer='he_normal')(input_layer)\n",
    "        x = SpatialDropout1D(self.spatial_dropout_rate)(x)\n",
    "        x = BatchNormalization()(x)\n",
    "        x = Activation('relu')(x)\n",
    "        \n",
    "        # Shortcut pathway\n",
    "        # Ensure the shortcut matches the dimension of the main pathway's output, adjust filters and stride as necessary\n",
    "        shortcut = Conv1D(N, 1, padding='same', kernel_initializer='he_normal')(input_layer)  # 1x1 conv for matching dimension\n",
    "        shortcut = BatchNormalization()(shortcut)  # Optional, for matching feature-wise statistics\n",
    "        \n",
    "        # Merging the shortcut with the main pathway\n",
    "        merged_output = Add()([x, shortcut])  # Element-wise addition\n",
    "\n",
    "        x = MaxPooling1D(pool_size=p)(merged_output)\n",
    "        \n",
    "        return x\n",
    "    \n",
    "    def get_spatial_layer(self, kernel_size, input_layer):\n",
    "        c = Conv1D(self.spatial_num_filters, kernel_size, kernel_initializer='he_normal')(input_layer)\n",
    "        # c = Conv1D(self.spatial_num_filters, kernel_size, data_format=\"channels_first\", kernel_initializer='he_normal')(input_layer)\n",
    "        b = tf.keras.layers.BatchNormalization()(c)\n",
    "        a = Activation('relu')(b)\n",
    "        do = SpatialDropout1D(0.1)(a)\n",
    "        return do\n",
    "\n",
    "    def build(self, input_shape=(5000, 12), fusion_shape=(1,)):\n",
    "        waveform_input = Input(shape=input_shape)\n",
    "        last_layer = waveform_input\n",
    "        \n",
    "        # Building CNN layers for waveform processing\n",
    "        for i in range(len(self.pool_sizes)):\n",
    "            if self.use_residual:\n",
    "                temp_layer = self.get_temporal_layer_with_residual(self.filter_numbers[i], self.kernel_widths[i],\n",
    "                                            self.pool_sizes[i], last_layer)\n",
    "            else:\n",
    "                temp_layer = self.get_temporal_layer(self.filter_numbers[i], self.kernel_widths[i],\n",
    "                                            self.pool_sizes[i], last_layer)\n",
    "            last_layer = temp_layer\n",
    "        \n",
    "        # last_layer = self.get_spatial_layer(input_shape[1], last_layer)\n",
    "        flattened_waveform = Flatten()(last_layer)\n",
    "\n",
    "        # Final Dense layers\n",
    "        x = Dense(64, activation='relu')(flattened_waveform)\n",
    "        x = Dropout(self.dropout_rate)(x)\n",
    "        x = Dense(32, activation='relu')(x)\n",
    "        x = Dropout(self.dropout_rate)(x)\n",
    "\n",
    "        fusion_input = Input(shape=fusion_shape)\n",
    "        # f = Dense(32, activation='relu')(fusion_input)\n",
    "        # f = Dropout(self.dropout_rate)(f)\n",
    "        # f = Dense(16, activation='relu')(f)\n",
    "        # f = Dropout(self.dropout_rate)(f)\n",
    "\n",
    "        if self.use_waveforms:\n",
    "            if self.fusion_strategy == \"concat\":\n",
    "                x = concatenate([x, fusion_input])\n",
    "                output = Dense(1, activation='sigmoid')(x)\n",
    "            \n",
    "            elif self.fusion_strategy == \"self_attn\":\n",
    "                # fusion_input = Dropout(self.dropout_rate)(fusion_input)\n",
    "                x = concatenate([x, fusion_input])\n",
    "\n",
    "                embed_dim = 16  # Dimensionality of the encoder.\n",
    "                num_heads = 4    # Number of attention heads.\n",
    "\n",
    "                cross_attention_layer = MultiHeadCrossAttention(embed_dim, num_heads)\n",
    "                x = cross_attention_layer(x, x)\n",
    "                x = Dropout(self.dropout_rate)(x)\n",
    "                output = Dense(1, activation='sigmoid')(x)\n",
    "\n",
    "            elif self.fusion_strategy == \"cross_attn\":\n",
    "                embed_dim = 16  # Dimensionality of the encoder.\n",
    "                num_heads = 4    # Number of attention heads.\n",
    "\n",
    "                cross_attention_layer = MultiHeadCrossAttention(embed_dim, num_heads)\n",
    "\n",
    "                # fusion_input = Dropout(self.dropout_rate)(fusion_input)\n",
    "                x = cross_attention_layer(x, fusion_input)\n",
    "                x = Dropout(self.dropout_rate)(x)\n",
    "                output = Dense(1, activation='sigmoid')(x)\n",
    "\n",
    "            if self.fusion_strategy == \"mlp\":\n",
    "                x = concatenate([x, fusion_input])\n",
    "                # WF/tab concat -> MLP -> output\n",
    "                x = Dense(32, activation='relu')(x)\n",
    "                x = Dropout(self.dropout_rate)(x)\n",
    "                x = Dense(16, activation='relu')(x)\n",
    "                x = Dropout(self.dropout_rate)(x)\n",
    "                x = Dense(8, activation='relu')(x)\n",
    "                x = Dropout(self.dropout_rate)(x)\n",
    "                output = Dense(1, activation='sigmoid')(x)\n",
    "\n",
    "            elif self.fusion_strategy == \"tab_mlp\":\n",
    "                x = Dense(16, activation='relu')(x)\n",
    "                x = Dropout(self.dropout_rate)(x)\n",
    "\n",
    "                fus = Dense(32, activation='relu')(fusion_input)\n",
    "                fus = Dropout(self.dropout_rate)(fus)\n",
    "                fus = Dense(16, activation='relu')(fus)\n",
    "                fus = Dropout(self.dropout_rate)(fus)\n",
    "\n",
    "                x = concatenate([x, fus])\n",
    "                output = Dense(1, activation='sigmoid')(x)\n",
    "        else:\n",
    "            x = fusion_input\n",
    "            output = Dense(1, activation='sigmoid')(x)\n",
    "\n",
    "            # x = Dense(16, activation='relu')(x)\n",
    "            # x = Dropout(self.dropout_rate)(x)\n",
    "            # x = Dense(8, activation='relu')(x)\n",
    "            # x = Dropout(self.dropout_rate)(x)\n",
    "\n",
    "        # x = Dense(16, activation='relu')(x)\n",
    "        # x = Dropout(self.dropout_rate)(x)\n",
    "        # x = concatenate([x, f])\n",
    "        # x = Dense(8, activation='relu')(x)\n",
    "\n",
    "        # WF/tab concat -> MLP -> output\n",
    "        # x = Dense(32, activation='relu')(x)\n",
    "        # x = Dropout(self.dropout_rate)(x)\n",
    "        # x = Dense(16, activation='relu')(x)\n",
    "        # x = Dropout(self.dropout_rate)(x)\n",
    "        # x = Dense(8, activation='relu')(x)\n",
    "        # x = Dropout(self.dropout_rate)(x)\n",
    "\n",
    "\n",
    "        # WF/tab concat -> attn -> output\n",
    "        # x = self.att([x, x])\n",
    "        # output = Dense(1, activation='sigmoid')(x)\n",
    "\n",
    "        self.model = Model(inputs=[waveform_input, fusion_input], outputs=output)\n",
    "\n",
    "        if self.verbose > 0:\n",
    "            print(self.model.summary())\n",
    "\n",
    "        return self.model\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "704be88b-c1b6-457b-970b-af3354676408",
   "metadata": {},
   "source": [
    "# Model training and evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "74d8e183-4a5b-42ff-9a96-726282992106",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Calculate steps per epoch\n",
    "steps_per_epoch = len(train_waveform_paths) // batch_size\n",
    "validation_steps = len(val_waveform_paths) // batch_size\n",
    "test_steps = len(test_waveform_paths) // batch_size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 172,
   "id": "9f2d727e-3d36-4994-afd4-688b33512086",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Attia et al. CNN model initialized with the following parameters:\n",
      "  filter_numbers: [16, 16, 32, 32, 64, 64]\n",
      "  kernel_widths: [5, 5, 5, 3, 3, 3]\n",
      "  pool_sizes: [2, 2, 4, 2, 2, 4]\n",
      "  spatial_num_filters: 64\n",
      "  dropout_rate: 0.2\n",
      "  dense_units: [64, 32]\n",
      "  fusion_strategy: concat\n",
      "  use_waveforms: True\n",
      "  use_residual: True\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\">Model: \"functional_9\"</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1mModel: \"functional_9\"\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">┏━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━┓\n",
       "┃<span style=\"font-weight: bold\"> Layer (type)        </span>┃<span style=\"font-weight: bold\"> Output Shape      </span>┃<span style=\"font-weight: bold\">    Param # </span>┃<span style=\"font-weight: bold\"> Connected to      </span>┃\n",
       "┡━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━┩\n",
       "│ input_layer_8       │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">5000</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">12</span>)  │          <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │ -                 │\n",
       "│ (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">InputLayer</span>)        │                   │            │                   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ conv1d_48 (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Conv1D</span>)  │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">5000</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">16</span>)  │        <span style=\"color: #00af00; text-decoration-color: #00af00\">976</span> │ input_layer_8[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>]… │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ spatial_dropout1d_… │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">5000</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">16</span>)  │          <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │ conv1d_48[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>][<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>]   │\n",
       "│ (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">SpatialDropout1D</span>)  │                   │            │                   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ batch_normalizatio… │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">5000</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">16</span>)  │         <span style=\"color: #00af00; text-decoration-color: #00af00\">64</span> │ spatial_dropout1… │\n",
       "│ (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">BatchNormalizatio…</span> │                   │            │                   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ conv1d_49 (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Conv1D</span>)  │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">5000</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">16</span>)  │        <span style=\"color: #00af00; text-decoration-color: #00af00\">208</span> │ input_layer_8[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>]… │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ activation_24       │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">5000</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">16</span>)  │          <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │ batch_normalizat… │\n",
       "│ (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Activation</span>)        │                   │            │                   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ batch_normalizatio… │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">5000</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">16</span>)  │         <span style=\"color: #00af00; text-decoration-color: #00af00\">64</span> │ conv1d_49[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>][<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>]   │\n",
       "│ (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">BatchNormalizatio…</span> │                   │            │                   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ add_24 (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Add</span>)        │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">5000</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">16</span>)  │          <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │ activation_24[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>]… │\n",
       "│                     │                   │            │ batch_normalizat… │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ max_pooling1d_24    │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">2500</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">16</span>)  │          <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │ add_24[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>][<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>]      │\n",
       "│ (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">MaxPooling1D</span>)      │                   │            │                   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ conv1d_50 (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Conv1D</span>)  │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">2500</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">16</span>)  │      <span style=\"color: #00af00; text-decoration-color: #00af00\">1,296</span> │ max_pooling1d_24… │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ spatial_dropout1d_… │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">2500</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">16</span>)  │          <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │ conv1d_50[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>][<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>]   │\n",
       "│ (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">SpatialDropout1D</span>)  │                   │            │                   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ batch_normalizatio… │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">2500</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">16</span>)  │         <span style=\"color: #00af00; text-decoration-color: #00af00\">64</span> │ spatial_dropout1… │\n",
       "│ (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">BatchNormalizatio…</span> │                   │            │                   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ conv1d_51 (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Conv1D</span>)  │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">2500</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">16</span>)  │        <span style=\"color: #00af00; text-decoration-color: #00af00\">272</span> │ max_pooling1d_24… │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ activation_25       │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">2500</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">16</span>)  │          <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │ batch_normalizat… │\n",
       "│ (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Activation</span>)        │                   │            │                   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ batch_normalizatio… │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">2500</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">16</span>)  │         <span style=\"color: #00af00; text-decoration-color: #00af00\">64</span> │ conv1d_51[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>][<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>]   │\n",
       "│ (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">BatchNormalizatio…</span> │                   │            │                   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ add_25 (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Add</span>)        │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">2500</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">16</span>)  │          <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │ activation_25[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>]… │\n",
       "│                     │                   │            │ batch_normalizat… │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ max_pooling1d_25    │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">1250</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">16</span>)  │          <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │ add_25[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>][<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>]      │\n",
       "│ (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">MaxPooling1D</span>)      │                   │            │                   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ conv1d_52 (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Conv1D</span>)  │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">1250</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">32</span>)  │      <span style=\"color: #00af00; text-decoration-color: #00af00\">2,592</span> │ max_pooling1d_25… │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ spatial_dropout1d_… │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">1250</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">32</span>)  │          <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │ conv1d_52[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>][<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>]   │\n",
       "│ (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">SpatialDropout1D</span>)  │                   │            │                   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ batch_normalizatio… │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">1250</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">32</span>)  │        <span style=\"color: #00af00; text-decoration-color: #00af00\">128</span> │ spatial_dropout1… │\n",
       "│ (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">BatchNormalizatio…</span> │                   │            │                   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ conv1d_53 (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Conv1D</span>)  │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">1250</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">32</span>)  │        <span style=\"color: #00af00; text-decoration-color: #00af00\">544</span> │ max_pooling1d_25… │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ activation_26       │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">1250</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">32</span>)  │          <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │ batch_normalizat… │\n",
       "│ (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Activation</span>)        │                   │            │                   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ batch_normalizatio… │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">1250</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">32</span>)  │        <span style=\"color: #00af00; text-decoration-color: #00af00\">128</span> │ conv1d_53[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>][<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>]   │\n",
       "│ (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">BatchNormalizatio…</span> │                   │            │                   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ add_26 (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Add</span>)        │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">1250</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">32</span>)  │          <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │ activation_26[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>]… │\n",
       "│                     │                   │            │ batch_normalizat… │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ max_pooling1d_26    │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">312</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">32</span>)   │          <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │ add_26[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>][<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>]      │\n",
       "│ (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">MaxPooling1D</span>)      │                   │            │                   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ conv1d_54 (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Conv1D</span>)  │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">312</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">32</span>)   │      <span style=\"color: #00af00; text-decoration-color: #00af00\">3,104</span> │ max_pooling1d_26… │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ spatial_dropout1d_… │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">312</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">32</span>)   │          <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │ conv1d_54[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>][<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>]   │\n",
       "│ (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">SpatialDropout1D</span>)  │                   │            │                   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ batch_normalizatio… │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">312</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">32</span>)   │        <span style=\"color: #00af00; text-decoration-color: #00af00\">128</span> │ spatial_dropout1… │\n",
       "│ (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">BatchNormalizatio…</span> │                   │            │                   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ conv1d_55 (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Conv1D</span>)  │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">312</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">32</span>)   │      <span style=\"color: #00af00; text-decoration-color: #00af00\">1,056</span> │ max_pooling1d_26… │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ activation_27       │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">312</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">32</span>)   │          <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │ batch_normalizat… │\n",
       "│ (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Activation</span>)        │                   │            │                   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ batch_normalizatio… │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">312</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">32</span>)   │        <span style=\"color: #00af00; text-decoration-color: #00af00\">128</span> │ conv1d_55[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>][<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>]   │\n",
       "│ (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">BatchNormalizatio…</span> │                   │            │                   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ add_27 (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Add</span>)        │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">312</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">32</span>)   │          <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │ activation_27[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>]… │\n",
       "│                     │                   │            │ batch_normalizat… │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ max_pooling1d_27    │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">156</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">32</span>)   │          <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │ add_27[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>][<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>]      │\n",
       "│ (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">MaxPooling1D</span>)      │                   │            │                   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ conv1d_56 (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Conv1D</span>)  │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">156</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">64</span>)   │      <span style=\"color: #00af00; text-decoration-color: #00af00\">6,208</span> │ max_pooling1d_27… │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ spatial_dropout1d_… │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">156</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">64</span>)   │          <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │ conv1d_56[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>][<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>]   │\n",
       "│ (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">SpatialDropout1D</span>)  │                   │            │                   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ batch_normalizatio… │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">156</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">64</span>)   │        <span style=\"color: #00af00; text-decoration-color: #00af00\">256</span> │ spatial_dropout1… │\n",
       "│ (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">BatchNormalizatio…</span> │                   │            │                   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ conv1d_57 (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Conv1D</span>)  │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">156</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">64</span>)   │      <span style=\"color: #00af00; text-decoration-color: #00af00\">2,112</span> │ max_pooling1d_27… │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ activation_28       │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">156</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">64</span>)   │          <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │ batch_normalizat… │\n",
       "│ (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Activation</span>)        │                   │            │                   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ batch_normalizatio… │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">156</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">64</span>)   │        <span style=\"color: #00af00; text-decoration-color: #00af00\">256</span> │ conv1d_57[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>][<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>]   │\n",
       "│ (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">BatchNormalizatio…</span> │                   │            │                   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ add_28 (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Add</span>)        │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">156</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">64</span>)   │          <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │ activation_28[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>]… │\n",
       "│                     │                   │            │ batch_normalizat… │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ max_pooling1d_28    │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">78</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">64</span>)    │          <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │ add_28[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>][<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>]      │\n",
       "│ (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">MaxPooling1D</span>)      │                   │            │                   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ conv1d_58 (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Conv1D</span>)  │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">78</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">64</span>)    │     <span style=\"color: #00af00; text-decoration-color: #00af00\">12,352</span> │ max_pooling1d_28… │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ spatial_dropout1d_… │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">78</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">64</span>)    │          <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │ conv1d_58[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>][<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>]   │\n",
       "│ (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">SpatialDropout1D</span>)  │                   │            │                   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ batch_normalizatio… │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">78</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">64</span>)    │        <span style=\"color: #00af00; text-decoration-color: #00af00\">256</span> │ spatial_dropout1… │\n",
       "│ (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">BatchNormalizatio…</span> │                   │            │                   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ conv1d_59 (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Conv1D</span>)  │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">78</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">64</span>)    │      <span style=\"color: #00af00; text-decoration-color: #00af00\">4,160</span> │ max_pooling1d_28… │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ activation_29       │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">78</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">64</span>)    │          <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │ batch_normalizat… │\n",
       "│ (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Activation</span>)        │                   │            │                   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ batch_normalizatio… │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">78</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">64</span>)    │        <span style=\"color: #00af00; text-decoration-color: #00af00\">256</span> │ conv1d_59[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>][<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>]   │\n",
       "│ (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">BatchNormalizatio…</span> │                   │            │                   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ add_29 (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Add</span>)        │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">78</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">64</span>)    │          <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │ activation_29[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>]… │\n",
       "│                     │                   │            │ batch_normalizat… │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ max_pooling1d_29    │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">19</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">64</span>)    │          <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │ add_29[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>][<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>]      │\n",
       "│ (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">MaxPooling1D</span>)      │                   │            │                   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ flatten_4 (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Flatten</span>) │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">1216</span>)      │          <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │ max_pooling1d_29… │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ dense_12 (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Dense</span>)    │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">64</span>)        │     <span style=\"color: #00af00; text-decoration-color: #00af00\">77,888</span> │ flatten_4[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>][<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>]   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ dropout_8 (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Dropout</span>) │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">64</span>)        │          <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │ dense_12[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>][<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>]    │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ dense_13 (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Dense</span>)    │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">32</span>)        │      <span style=\"color: #00af00; text-decoration-color: #00af00\">2,080</span> │ dropout_8[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>][<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>]   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ dropout_9 (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Dropout</span>) │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">32</span>)        │          <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │ dense_13[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>][<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>]    │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ input_layer_9       │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">10</span>)        │          <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │ -                 │\n",
       "│ (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">InputLayer</span>)        │                   │            │                   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ concatenate_4       │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">42</span>)        │          <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │ dropout_9[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>][<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>],  │\n",
       "│ (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Concatenate</span>)       │                   │            │ input_layer_9[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>]… │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ dense_14 (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Dense</span>)    │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">1</span>)         │         <span style=\"color: #00af00; text-decoration-color: #00af00\">43</span> │ concatenate_4[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>]… │\n",
       "└─────────────────────┴───────────────────┴────────────┴───────────────────┘\n",
       "</pre>\n"
      ],
      "text/plain": [
       "┏━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━┓\n",
       "┃\u001b[1m \u001b[0m\u001b[1mLayer (type)       \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mOutput Shape     \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1m   Param #\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mConnected to     \u001b[0m\u001b[1m \u001b[0m┃\n",
       "┡━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━┩\n",
       "│ input_layer_8       │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m5000\u001b[0m, \u001b[38;5;34m12\u001b[0m)  │          \u001b[38;5;34m0\u001b[0m │ -                 │\n",
       "│ (\u001b[38;5;33mInputLayer\u001b[0m)        │                   │            │                   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ conv1d_48 (\u001b[38;5;33mConv1D\u001b[0m)  │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m5000\u001b[0m, \u001b[38;5;34m16\u001b[0m)  │        \u001b[38;5;34m976\u001b[0m │ input_layer_8[\u001b[38;5;34m0\u001b[0m]… │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ spatial_dropout1d_… │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m5000\u001b[0m, \u001b[38;5;34m16\u001b[0m)  │          \u001b[38;5;34m0\u001b[0m │ conv1d_48[\u001b[38;5;34m0\u001b[0m][\u001b[38;5;34m0\u001b[0m]   │\n",
       "│ (\u001b[38;5;33mSpatialDropout1D\u001b[0m)  │                   │            │                   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ batch_normalizatio… │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m5000\u001b[0m, \u001b[38;5;34m16\u001b[0m)  │         \u001b[38;5;34m64\u001b[0m │ spatial_dropout1… │\n",
       "│ (\u001b[38;5;33mBatchNormalizatio…\u001b[0m │                   │            │                   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ conv1d_49 (\u001b[38;5;33mConv1D\u001b[0m)  │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m5000\u001b[0m, \u001b[38;5;34m16\u001b[0m)  │        \u001b[38;5;34m208\u001b[0m │ input_layer_8[\u001b[38;5;34m0\u001b[0m]… │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ activation_24       │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m5000\u001b[0m, \u001b[38;5;34m16\u001b[0m)  │          \u001b[38;5;34m0\u001b[0m │ batch_normalizat… │\n",
       "│ (\u001b[38;5;33mActivation\u001b[0m)        │                   │            │                   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ batch_normalizatio… │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m5000\u001b[0m, \u001b[38;5;34m16\u001b[0m)  │         \u001b[38;5;34m64\u001b[0m │ conv1d_49[\u001b[38;5;34m0\u001b[0m][\u001b[38;5;34m0\u001b[0m]   │\n",
       "│ (\u001b[38;5;33mBatchNormalizatio…\u001b[0m │                   │            │                   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ add_24 (\u001b[38;5;33mAdd\u001b[0m)        │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m5000\u001b[0m, \u001b[38;5;34m16\u001b[0m)  │          \u001b[38;5;34m0\u001b[0m │ activation_24[\u001b[38;5;34m0\u001b[0m]… │\n",
       "│                     │                   │            │ batch_normalizat… │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ max_pooling1d_24    │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m2500\u001b[0m, \u001b[38;5;34m16\u001b[0m)  │          \u001b[38;5;34m0\u001b[0m │ add_24[\u001b[38;5;34m0\u001b[0m][\u001b[38;5;34m0\u001b[0m]      │\n",
       "│ (\u001b[38;5;33mMaxPooling1D\u001b[0m)      │                   │            │                   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ conv1d_50 (\u001b[38;5;33mConv1D\u001b[0m)  │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m2500\u001b[0m, \u001b[38;5;34m16\u001b[0m)  │      \u001b[38;5;34m1,296\u001b[0m │ max_pooling1d_24… │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ spatial_dropout1d_… │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m2500\u001b[0m, \u001b[38;5;34m16\u001b[0m)  │          \u001b[38;5;34m0\u001b[0m │ conv1d_50[\u001b[38;5;34m0\u001b[0m][\u001b[38;5;34m0\u001b[0m]   │\n",
       "│ (\u001b[38;5;33mSpatialDropout1D\u001b[0m)  │                   │            │                   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ batch_normalizatio… │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m2500\u001b[0m, \u001b[38;5;34m16\u001b[0m)  │         \u001b[38;5;34m64\u001b[0m │ spatial_dropout1… │\n",
       "│ (\u001b[38;5;33mBatchNormalizatio…\u001b[0m │                   │            │                   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ conv1d_51 (\u001b[38;5;33mConv1D\u001b[0m)  │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m2500\u001b[0m, \u001b[38;5;34m16\u001b[0m)  │        \u001b[38;5;34m272\u001b[0m │ max_pooling1d_24… │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ activation_25       │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m2500\u001b[0m, \u001b[38;5;34m16\u001b[0m)  │          \u001b[38;5;34m0\u001b[0m │ batch_normalizat… │\n",
       "│ (\u001b[38;5;33mActivation\u001b[0m)        │                   │            │                   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ batch_normalizatio… │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m2500\u001b[0m, \u001b[38;5;34m16\u001b[0m)  │         \u001b[38;5;34m64\u001b[0m │ conv1d_51[\u001b[38;5;34m0\u001b[0m][\u001b[38;5;34m0\u001b[0m]   │\n",
       "│ (\u001b[38;5;33mBatchNormalizatio…\u001b[0m │                   │            │                   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ add_25 (\u001b[38;5;33mAdd\u001b[0m)        │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m2500\u001b[0m, \u001b[38;5;34m16\u001b[0m)  │          \u001b[38;5;34m0\u001b[0m │ activation_25[\u001b[38;5;34m0\u001b[0m]… │\n",
       "│                     │                   │            │ batch_normalizat… │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ max_pooling1d_25    │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m1250\u001b[0m, \u001b[38;5;34m16\u001b[0m)  │          \u001b[38;5;34m0\u001b[0m │ add_25[\u001b[38;5;34m0\u001b[0m][\u001b[38;5;34m0\u001b[0m]      │\n",
       "│ (\u001b[38;5;33mMaxPooling1D\u001b[0m)      │                   │            │                   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ conv1d_52 (\u001b[38;5;33mConv1D\u001b[0m)  │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m1250\u001b[0m, \u001b[38;5;34m32\u001b[0m)  │      \u001b[38;5;34m2,592\u001b[0m │ max_pooling1d_25… │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ spatial_dropout1d_… │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m1250\u001b[0m, \u001b[38;5;34m32\u001b[0m)  │          \u001b[38;5;34m0\u001b[0m │ conv1d_52[\u001b[38;5;34m0\u001b[0m][\u001b[38;5;34m0\u001b[0m]   │\n",
       "│ (\u001b[38;5;33mSpatialDropout1D\u001b[0m)  │                   │            │                   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ batch_normalizatio… │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m1250\u001b[0m, \u001b[38;5;34m32\u001b[0m)  │        \u001b[38;5;34m128\u001b[0m │ spatial_dropout1… │\n",
       "│ (\u001b[38;5;33mBatchNormalizatio…\u001b[0m │                   │            │                   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ conv1d_53 (\u001b[38;5;33mConv1D\u001b[0m)  │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m1250\u001b[0m, \u001b[38;5;34m32\u001b[0m)  │        \u001b[38;5;34m544\u001b[0m │ max_pooling1d_25… │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ activation_26       │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m1250\u001b[0m, \u001b[38;5;34m32\u001b[0m)  │          \u001b[38;5;34m0\u001b[0m │ batch_normalizat… │\n",
       "│ (\u001b[38;5;33mActivation\u001b[0m)        │                   │            │                   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ batch_normalizatio… │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m1250\u001b[0m, \u001b[38;5;34m32\u001b[0m)  │        \u001b[38;5;34m128\u001b[0m │ conv1d_53[\u001b[38;5;34m0\u001b[0m][\u001b[38;5;34m0\u001b[0m]   │\n",
       "│ (\u001b[38;5;33mBatchNormalizatio…\u001b[0m │                   │            │                   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ add_26 (\u001b[38;5;33mAdd\u001b[0m)        │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m1250\u001b[0m, \u001b[38;5;34m32\u001b[0m)  │          \u001b[38;5;34m0\u001b[0m │ activation_26[\u001b[38;5;34m0\u001b[0m]… │\n",
       "│                     │                   │            │ batch_normalizat… │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ max_pooling1d_26    │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m312\u001b[0m, \u001b[38;5;34m32\u001b[0m)   │          \u001b[38;5;34m0\u001b[0m │ add_26[\u001b[38;5;34m0\u001b[0m][\u001b[38;5;34m0\u001b[0m]      │\n",
       "│ (\u001b[38;5;33mMaxPooling1D\u001b[0m)      │                   │            │                   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ conv1d_54 (\u001b[38;5;33mConv1D\u001b[0m)  │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m312\u001b[0m, \u001b[38;5;34m32\u001b[0m)   │      \u001b[38;5;34m3,104\u001b[0m │ max_pooling1d_26… │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ spatial_dropout1d_… │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m312\u001b[0m, \u001b[38;5;34m32\u001b[0m)   │          \u001b[38;5;34m0\u001b[0m │ conv1d_54[\u001b[38;5;34m0\u001b[0m][\u001b[38;5;34m0\u001b[0m]   │\n",
       "│ (\u001b[38;5;33mSpatialDropout1D\u001b[0m)  │                   │            │                   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ batch_normalizatio… │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m312\u001b[0m, \u001b[38;5;34m32\u001b[0m)   │        \u001b[38;5;34m128\u001b[0m │ spatial_dropout1… │\n",
       "│ (\u001b[38;5;33mBatchNormalizatio…\u001b[0m │                   │            │                   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ conv1d_55 (\u001b[38;5;33mConv1D\u001b[0m)  │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m312\u001b[0m, \u001b[38;5;34m32\u001b[0m)   │      \u001b[38;5;34m1,056\u001b[0m │ max_pooling1d_26… │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ activation_27       │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m312\u001b[0m, \u001b[38;5;34m32\u001b[0m)   │          \u001b[38;5;34m0\u001b[0m │ batch_normalizat… │\n",
       "│ (\u001b[38;5;33mActivation\u001b[0m)        │                   │            │                   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ batch_normalizatio… │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m312\u001b[0m, \u001b[38;5;34m32\u001b[0m)   │        \u001b[38;5;34m128\u001b[0m │ conv1d_55[\u001b[38;5;34m0\u001b[0m][\u001b[38;5;34m0\u001b[0m]   │\n",
       "│ (\u001b[38;5;33mBatchNormalizatio…\u001b[0m │                   │            │                   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ add_27 (\u001b[38;5;33mAdd\u001b[0m)        │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m312\u001b[0m, \u001b[38;5;34m32\u001b[0m)   │          \u001b[38;5;34m0\u001b[0m │ activation_27[\u001b[38;5;34m0\u001b[0m]… │\n",
       "│                     │                   │            │ batch_normalizat… │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ max_pooling1d_27    │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m156\u001b[0m, \u001b[38;5;34m32\u001b[0m)   │          \u001b[38;5;34m0\u001b[0m │ add_27[\u001b[38;5;34m0\u001b[0m][\u001b[38;5;34m0\u001b[0m]      │\n",
       "│ (\u001b[38;5;33mMaxPooling1D\u001b[0m)      │                   │            │                   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ conv1d_56 (\u001b[38;5;33mConv1D\u001b[0m)  │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m156\u001b[0m, \u001b[38;5;34m64\u001b[0m)   │      \u001b[38;5;34m6,208\u001b[0m │ max_pooling1d_27… │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ spatial_dropout1d_… │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m156\u001b[0m, \u001b[38;5;34m64\u001b[0m)   │          \u001b[38;5;34m0\u001b[0m │ conv1d_56[\u001b[38;5;34m0\u001b[0m][\u001b[38;5;34m0\u001b[0m]   │\n",
       "│ (\u001b[38;5;33mSpatialDropout1D\u001b[0m)  │                   │            │                   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ batch_normalizatio… │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m156\u001b[0m, \u001b[38;5;34m64\u001b[0m)   │        \u001b[38;5;34m256\u001b[0m │ spatial_dropout1… │\n",
       "│ (\u001b[38;5;33mBatchNormalizatio…\u001b[0m │                   │            │                   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ conv1d_57 (\u001b[38;5;33mConv1D\u001b[0m)  │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m156\u001b[0m, \u001b[38;5;34m64\u001b[0m)   │      \u001b[38;5;34m2,112\u001b[0m │ max_pooling1d_27… │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ activation_28       │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m156\u001b[0m, \u001b[38;5;34m64\u001b[0m)   │          \u001b[38;5;34m0\u001b[0m │ batch_normalizat… │\n",
       "│ (\u001b[38;5;33mActivation\u001b[0m)        │                   │            │                   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ batch_normalizatio… │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m156\u001b[0m, \u001b[38;5;34m64\u001b[0m)   │        \u001b[38;5;34m256\u001b[0m │ conv1d_57[\u001b[38;5;34m0\u001b[0m][\u001b[38;5;34m0\u001b[0m]   │\n",
       "│ (\u001b[38;5;33mBatchNormalizatio…\u001b[0m │                   │            │                   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ add_28 (\u001b[38;5;33mAdd\u001b[0m)        │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m156\u001b[0m, \u001b[38;5;34m64\u001b[0m)   │          \u001b[38;5;34m0\u001b[0m │ activation_28[\u001b[38;5;34m0\u001b[0m]… │\n",
       "│                     │                   │            │ batch_normalizat… │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ max_pooling1d_28    │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m78\u001b[0m, \u001b[38;5;34m64\u001b[0m)    │          \u001b[38;5;34m0\u001b[0m │ add_28[\u001b[38;5;34m0\u001b[0m][\u001b[38;5;34m0\u001b[0m]      │\n",
       "│ (\u001b[38;5;33mMaxPooling1D\u001b[0m)      │                   │            │                   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ conv1d_58 (\u001b[38;5;33mConv1D\u001b[0m)  │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m78\u001b[0m, \u001b[38;5;34m64\u001b[0m)    │     \u001b[38;5;34m12,352\u001b[0m │ max_pooling1d_28… │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ spatial_dropout1d_… │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m78\u001b[0m, \u001b[38;5;34m64\u001b[0m)    │          \u001b[38;5;34m0\u001b[0m │ conv1d_58[\u001b[38;5;34m0\u001b[0m][\u001b[38;5;34m0\u001b[0m]   │\n",
       "│ (\u001b[38;5;33mSpatialDropout1D\u001b[0m)  │                   │            │                   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ batch_normalizatio… │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m78\u001b[0m, \u001b[38;5;34m64\u001b[0m)    │        \u001b[38;5;34m256\u001b[0m │ spatial_dropout1… │\n",
       "│ (\u001b[38;5;33mBatchNormalizatio…\u001b[0m │                   │            │                   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ conv1d_59 (\u001b[38;5;33mConv1D\u001b[0m)  │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m78\u001b[0m, \u001b[38;5;34m64\u001b[0m)    │      \u001b[38;5;34m4,160\u001b[0m │ max_pooling1d_28… │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ activation_29       │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m78\u001b[0m, \u001b[38;5;34m64\u001b[0m)    │          \u001b[38;5;34m0\u001b[0m │ batch_normalizat… │\n",
       "│ (\u001b[38;5;33mActivation\u001b[0m)        │                   │            │                   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ batch_normalizatio… │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m78\u001b[0m, \u001b[38;5;34m64\u001b[0m)    │        \u001b[38;5;34m256\u001b[0m │ conv1d_59[\u001b[38;5;34m0\u001b[0m][\u001b[38;5;34m0\u001b[0m]   │\n",
       "│ (\u001b[38;5;33mBatchNormalizatio…\u001b[0m │                   │            │                   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ add_29 (\u001b[38;5;33mAdd\u001b[0m)        │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m78\u001b[0m, \u001b[38;5;34m64\u001b[0m)    │          \u001b[38;5;34m0\u001b[0m │ activation_29[\u001b[38;5;34m0\u001b[0m]… │\n",
       "│                     │                   │            │ batch_normalizat… │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ max_pooling1d_29    │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m19\u001b[0m, \u001b[38;5;34m64\u001b[0m)    │          \u001b[38;5;34m0\u001b[0m │ add_29[\u001b[38;5;34m0\u001b[0m][\u001b[38;5;34m0\u001b[0m]      │\n",
       "│ (\u001b[38;5;33mMaxPooling1D\u001b[0m)      │                   │            │                   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ flatten_4 (\u001b[38;5;33mFlatten\u001b[0m) │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m1216\u001b[0m)      │          \u001b[38;5;34m0\u001b[0m │ max_pooling1d_29… │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ dense_12 (\u001b[38;5;33mDense\u001b[0m)    │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m64\u001b[0m)        │     \u001b[38;5;34m77,888\u001b[0m │ flatten_4[\u001b[38;5;34m0\u001b[0m][\u001b[38;5;34m0\u001b[0m]   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ dropout_8 (\u001b[38;5;33mDropout\u001b[0m) │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m64\u001b[0m)        │          \u001b[38;5;34m0\u001b[0m │ dense_12[\u001b[38;5;34m0\u001b[0m][\u001b[38;5;34m0\u001b[0m]    │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ dense_13 (\u001b[38;5;33mDense\u001b[0m)    │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m32\u001b[0m)        │      \u001b[38;5;34m2,080\u001b[0m │ dropout_8[\u001b[38;5;34m0\u001b[0m][\u001b[38;5;34m0\u001b[0m]   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ dropout_9 (\u001b[38;5;33mDropout\u001b[0m) │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m32\u001b[0m)        │          \u001b[38;5;34m0\u001b[0m │ dense_13[\u001b[38;5;34m0\u001b[0m][\u001b[38;5;34m0\u001b[0m]    │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ input_layer_9       │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m10\u001b[0m)        │          \u001b[38;5;34m0\u001b[0m │ -                 │\n",
       "│ (\u001b[38;5;33mInputLayer\u001b[0m)        │                   │            │                   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ concatenate_4       │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m42\u001b[0m)        │          \u001b[38;5;34m0\u001b[0m │ dropout_9[\u001b[38;5;34m0\u001b[0m][\u001b[38;5;34m0\u001b[0m],  │\n",
       "│ (\u001b[38;5;33mConcatenate\u001b[0m)       │                   │            │ input_layer_9[\u001b[38;5;34m0\u001b[0m]… │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ dense_14 (\u001b[38;5;33mDense\u001b[0m)    │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m1\u001b[0m)         │         \u001b[38;5;34m43\u001b[0m │ concatenate_4[\u001b[38;5;34m0\u001b[0m]… │\n",
       "└─────────────────────┴───────────────────┴────────────┴───────────────────┘\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\"> Total params: </span><span style=\"color: #00af00; text-decoration-color: #00af00\">116,683</span> (455.79 KB)\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1m Total params: \u001b[0m\u001b[38;5;34m116,683\u001b[0m (455.79 KB)\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\"> Trainable params: </span><span style=\"color: #00af00; text-decoration-color: #00af00\">115,787</span> (452.29 KB)\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1m Trainable params: \u001b[0m\u001b[38;5;34m115,787\u001b[0m (452.29 KB)\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\"> Non-trainable params: </span><span style=\"color: #00af00; text-decoration-color: #00af00\">896</span> (3.50 KB)\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1m Non-trainable params: \u001b[0m\u001b[38;5;34m896\u001b[0m (3.50 KB)\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "None\n",
      "Epoch 1/20\n",
      "\u001b[1m200/200\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m668s\u001b[0m 3s/step - accuracy: 0.7081 - auc: 0.5384 - loss: 0.6457 - val_accuracy: 0.7713 - val_auc: 0.7270 - val_loss: 0.4987 - learning_rate: 0.0010\n",
      "Epoch 2/20\n",
      "\u001b[1m200/200\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m622s\u001b[0m 3s/step - accuracy: 0.7689 - auc: 0.6635 - loss: 0.5218 - val_accuracy: 0.7715 - val_auc: 0.7542 - val_loss: 0.4824 - learning_rate: 0.0010\n",
      "Epoch 3/20\n",
      "\u001b[1m200/200\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m624s\u001b[0m 3s/step - accuracy: 0.7800 - auc: 0.6974 - loss: 0.5016 - val_accuracy: 0.7755 - val_auc: 0.7617 - val_loss: 0.4734 - learning_rate: 0.0010\n",
      "Epoch 4/20\n",
      "\u001b[1m200/200\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m626s\u001b[0m 3s/step - accuracy: 0.7752 - auc: 0.7217 - loss: 0.4901 - val_accuracy: 0.7767 - val_auc: 0.7644 - val_loss: 0.4692 - learning_rate: 0.0010\n",
      "Epoch 5/20\n",
      "\u001b[1m200/200\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m627s\u001b[0m 3s/step - accuracy: 0.7808 - auc: 0.7482 - loss: 0.4734 - val_accuracy: 0.7769 - val_auc: 0.7670 - val_loss: 0.4666 - learning_rate: 0.0010\n",
      "Epoch 6/20\n",
      "\u001b[1m200/200\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m637s\u001b[0m 3s/step - accuracy: 0.7816 - auc: 0.7474 - loss: 0.4731 - val_accuracy: 0.7794 - val_auc: 0.7644 - val_loss: 0.4702 - learning_rate: 0.0010\n",
      "Epoch 7/20\n",
      "\u001b[1m200/200\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m651s\u001b[0m 3s/step - accuracy: 0.7881 - auc: 0.7610 - loss: 0.4630 - val_accuracy: 0.7766 - val_auc: 0.7694 - val_loss: 0.4685 - learning_rate: 0.0010\n",
      "Epoch 8/20\n",
      "\u001b[1m200/200\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m552s\u001b[0m 3s/step - accuracy: 0.7807 - auc: 0.7702 - loss: 0.4603 - val_accuracy: 0.7773 - val_auc: 0.7643 - val_loss: 0.4777 - learning_rate: 0.0010\n",
      "Epoch 9/20\n",
      "\u001b[1m200/200\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m551s\u001b[0m 3s/step - accuracy: 0.7721 - auc: 0.7646 - loss: 0.4659 - val_accuracy: 0.7781 - val_auc: 0.7699 - val_loss: 0.4654 - learning_rate: 0.0010\n",
      "Epoch 10/20\n",
      "\u001b[1m200/200\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m552s\u001b[0m 3s/step - accuracy: 0.7786 - auc: 0.7790 - loss: 0.4504 - val_accuracy: 0.7767 - val_auc: 0.7732 - val_loss: 0.4660 - learning_rate: 0.0010\n",
      "Epoch 11/20\n",
      "\u001b[1m200/200\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m559s\u001b[0m 3s/step - accuracy: 0.7751 - auc: 0.7763 - loss: 0.4576 - val_accuracy: 0.7811 - val_auc: 0.7722 - val_loss: 0.4632 - learning_rate: 0.0010\n",
      "Epoch 12/20\n",
      "\u001b[1m200/200\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m554s\u001b[0m 3s/step - accuracy: 0.7819 - auc: 0.7865 - loss: 0.4497 - val_accuracy: 0.7780 - val_auc: 0.7676 - val_loss: 0.4673 - learning_rate: 0.0010\n",
      "Epoch 13/20\n",
      "\u001b[1m200/200\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m556s\u001b[0m 3s/step - accuracy: 0.7787 - auc: 0.7807 - loss: 0.4555 - val_accuracy: 0.7792 - val_auc: 0.7602 - val_loss: 0.4736 - learning_rate: 0.0010\n",
      "Epoch 14/20\n",
      "\u001b[1m200/200\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m554s\u001b[0m 3s/step - accuracy: 0.7785 - auc: 0.7761 - loss: 0.4587 - val_accuracy: 0.7784 - val_auc: 0.7644 - val_loss: 0.4697 - learning_rate: 0.0010\n",
      "Epoch 15/20\n",
      "\u001b[1m200/200\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m556s\u001b[0m 3s/step - accuracy: 0.7848 - auc: 0.7876 - loss: 0.4498 - val_accuracy: 0.7827 - val_auc: 0.7678 - val_loss: 0.4668 - learning_rate: 0.0010\n",
      "Epoch 16/20\n",
      "\u001b[1m200/200\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m551s\u001b[0m 3s/step - accuracy: 0.7861 - auc: 0.7930 - loss: 0.4450 - val_accuracy: 0.7800 - val_auc: 0.7658 - val_loss: 0.4721 - learning_rate: 0.0010\n",
      "Epoch 17/20\n",
      "\u001b[1m200/200\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m550s\u001b[0m 3s/step - accuracy: 0.7871 - auc: 0.8003 - loss: 0.4384 - val_accuracy: 0.7788 - val_auc: 0.7631 - val_loss: 0.4709 - learning_rate: 0.0010\n",
      "Epoch 18/20\n",
      "\u001b[1m200/200\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m552s\u001b[0m 3s/step - accuracy: 0.7858 - auc: 0.7933 - loss: 0.4492 - val_accuracy: 0.7766 - val_auc: 0.7585 - val_loss: 0.4794 - learning_rate: 0.0010\n",
      "Epoch 19/20\n",
      "\u001b[1m200/200\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m554s\u001b[0m 3s/step - accuracy: 0.7901 - auc: 0.8052 - loss: 0.4405 - val_accuracy: 0.7805 - val_auc: 0.7664 - val_loss: 0.4716 - learning_rate: 0.0010\n",
      "Epoch 20/20\n",
      "\u001b[1m200/200\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m553s\u001b[0m 3s/step - accuracy: 0.7894 - auc: 0.8093 - loss: 0.4358 - val_accuracy: 0.7775 - val_auc: 0.7610 - val_loss: 0.4743 - learning_rate: 0.0010\n"
     ]
    }
   ],
   "source": [
    "\n",
    "# Define and build the model\n",
    "fusion_model = Attia_et_al_fusion()\n",
    "model = fusion_model.build(input_shape=(5000, 12), fusion_shape=(train_tab_data.shape[1],))\n",
    "\n",
    "# Compile the model\n",
    "model.compile(optimizer='adam', \n",
    "              loss='binary_crossentropy', \n",
    "              metrics=['accuracy', AUC(name='auc')])\n",
    "\n",
    "# Training parameters\n",
    "EPOCHS = 50\n",
    "\n",
    "# Callbacks (example)\n",
    "reduce_lr = tf.keras.callbacks.ReduceLROnPlateau(monitor='val_loss', factor=0.2, patience=5, min_lr=0.001)\n",
    "early_stopping = tf.keras.callbacks.EarlyStopping(monitor='val_loss', patience=10, restore_best_weights=True)\n",
    "\n",
    "# Train the model\n",
    "history = model.fit(\n",
    "    train_dataset,\n",
    "    validation_data=val_dataset,\n",
    "    epochs=EPOCHS,\n",
    "    steps_per_epoch=steps_per_epoch,\n",
    "    validation_steps=validation_steps,\n",
    "    callbacks=[reduce_lr, early_stopping],\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a0a2bd02-2f8c-478e-b210-8754bec0dc47",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save the trained model\n",
    "model.save('trained_fusion_model.keras')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "id": "c64526b0-ff04-4879-884e-d4b08bb8f0b8",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/idies/miniconda3/lib/python3.9/site-packages/keras/src/saving/saving_lib.py:415: UserWarning: Skipping variable loading for optimizer 'rmsprop', because it has 56 variables whereas the saved optimizer has 110 variables. \n",
      "  saveable.load_own_variables(weights_store.get(inner_path))\n"
     ]
    }
   ],
   "source": [
    "trained_model = tf.keras.models.load_model('trained_fusion_model.keras', custom_objects={'AUC': tf.keras.metrics.AUC})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "5012e024-1de2-44d5-887d-6544929916f7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1m171/171\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m370s\u001b[0m 2s/step - accuracy: 0.7855 - auc: 0.7844 - loss: 0.4533\n",
      "Test Loss: 0.463108092546463\n",
      "Test Accuracy: 0.7799707651138306\n",
      "Test AUC: 0.7763502597808838\n"
     ]
    }
   ],
   "source": [
    "# Evaluate the model on the test dataset\n",
    "test_loss, test_accuracy, test_auc = trained_model.evaluate(test_dataset, steps=test_steps)\n",
    "print(f\"Test Loss: {test_loss}\")\n",
    "print(f\"Test Accuracy: {test_accuracy}\")\n",
    "print(f\"Test AUC: {test_auc}\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.17"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
